#!/usr/bin/python -tt
# -*- coding: utf-8 -*-
from urllib2 import *
from urllib import *
from bs4 import BeautifulSoup,Tag,NavigableString
import re
import json
from pprint import pprint
from itertools import cycle
from decimal import Decimal
from datetime import datetime

SEARCH_URL =  "https://ug3.technion.ac.il/rishum/search"
COURSE_URL =  "https://ug3.technion.ac.il/rishum/course?MK=%s&SEM=%s"
TEST_URL = "https://ug3.technion.ac.il/rishum/exams/%s/%s"
QUIZ_URL = "https://ug3.technion.ac.il/rishum/quizzes/%s/%s"

def __get_exams_location_string(tags):
    string = ""
    for y in tags:
        if isinstance(y, NavigableString):
            string = string + y + ", "
    return string[:-2]

def __get_group_lect_list(column):
    ret_list =[]
    for z in column.contents:
        if z.name == "a":
            ret_list.append((z.text, z["href"]))
    return ret_list

def __get_group_data_list(column):
    ret_list =[]
    for z in column.contents:
        if isinstance(z, NavigableString):
            ret_list.append(z)
    return ret_list



def __get_all_facs_htmls(sem):
    """return [(Faculty name, Faculty courses HTML, Faculty num),...]"""
    search_site_html = urlopen(SEARCH_URL).read()
    soup = BeautifulSoup(search_site_html)
    
    facs = []
    for fac in soup.find("select",attrs={"name":"FAC"}).children:
        if type(fac).__name__ == "Tag":
            if fac["value"] != "":
                facs.append((fac.string.strip(),fac["value"]))
    
    facs_htmls = []
    for fac in facs:
        data = urlencode({"FAC":fac[1],"SEM":sem,"RECALL":"Y","D1":"on","D2":"on","D3":"on","D4":"on","D5":"on","D6":"on","OPTSTUD":"on","doSearch":"Y","Search":"   חפש "})
        facs_htmls.append((fac[0], urlopen(SEARCH_URL,data).read(),fac[1]))
    return facs_htmls

def __get_all_facs_data(sem):
    """return {(Faculty name,Faculty num): [{"name": course1_name, "num" : course1_num},...]}"""
    faculties_data = {}
    for fac in __get_all_facs_htmls(sem):
        soup = BeautifulSoup(fac[1])
        courses = soup.find_all("div",{"class":"result-row"})
        fac_courses_list = []
        for course_div in courses:
            course = course_div.contents
            fac_courses_list.append({"num":course[1].a.string,"name":course[3].string.strip()})
        faculties_data[(fac[0],fac[2])] = fac_courses_list
    return faculties_data

def __get_course_url(soup):
    url_tag = soup.find("div",{"class":"property"},text = re.compile(u"אתר הקורס"))
    if url_tag is None:
        return None 
    return url_tag.next_sibling.next_sibling.contents[1]["href"]

def __get_course_points(soup):
    return soup.find("div",{"class":"property"},text = re.compile(u"נקודות")).next_sibling.next_sibling.string.strip()
    
def __get_course_types_hours(soup):
    """return [(classType, hours),...,totalHours]"""
    class_types_list = []
    totalHours = Decimal(0)
    for tag in soup.find("div",{"class":"properties-section"},text = re.compile(u"שעות שבועיות")).next_siblings:
        if isinstance(tag,Tag):
            if tag["class"][0] == "property":
                class_type = tag.string.strip()
            elif tag["class"][0] == "property-value":
                class_types_list.append((class_type,tag.string.strip()))
                totalHours = totalHours + Decimal(tag.string.strip())
            else:
                break
    return class_types_list + [totalHours.to_eng_string()]

def __get_course_syllabus(soup):
    return soup.find("div",{"class":"property"},text = re.compile(u"סילבוס")).next_sibling.next_sibling.string.strip()

def __get_course_prerequisites(soup):
    """return [[course1, (and) course2, ...] ,(or) []...]"""
    curr_pre_courses_together = [[]]
    pre_tag = soup.find("div",{"class":"property"},text = re.compile(u"מקצועות קדם"))
    if pre_tag is not None: 
        for x in pre_tag.next_sibling.next_sibling.children:
            if isinstance(x,Tag):
                curr_pre_courses_together[-1].append((x.string,x["data-content"]))
            if isinstance(x,NavigableString) and u"או" in x:
                curr_pre_courses_together.append([])
    return curr_pre_courses_together

def __get_course_notes(soup):
    return soup.find("div",{"class":"property"},text = re.compile(u"הערות")).next_sibling.next_sibling.text.strip()

def __get_course_exams(soup):
    """return {"a" : {"date" : a_date, "times" : a_times, "notes" : a_notes, "locations" : a_locations}, 
            "b" : {"date" : b_date, "times" : b_times, "notes" : b_notes, "locations" : b_locations}} """
    exam_tag = soup.find("div",{"class":"properties-wrapper"})
    if exam_tag is None:
        return None
    
    c = exam_tag.contents
    a_date = c[4].text[:10]
    a_times = __format_time(c[8].text)
    a_notes = c[12].text
    a_locations = __get_exams_location_string(c[16])
    
    try :
        b_date = c[22].text[:10]
        b_times = __format_time(c[26].text)
        b_notes = c[30].text
        b_locations = __get_exams_location_string(c[34])
    
    except IndexError:
        return {"a" : {"date" : a_date, "times" : a_times, "notes" : a_notes, "locations" : a_locations}}
         
            
    
    return {"a" : {"date" : a_date, "times" : a_times, "notes" : a_notes, "locations" : a_locations}, 
            "b" : {"date" : b_date, "times" : b_times, "notes" : b_notes, "locations" : b_locations}}

def __get_course_quizzes(soup):
    """return {quizName : {"date" : date,
                                    "times" : times, 
                                    "notes" : notes, 
                                    "locations" : locations}"""
    quizzes_dict = {}
    for x in soup.find_all("div",{"class":"properties-section"}):
        c = list(x.next_siblings)
        quizzes_dict[x.text.strip()] = {"date" : c[3].text[:10],
                                    "times" : c[7].text, 
                                    "notes" : c[11].text, 
                                    "locations" : __get_exams_location_string(c[15])}
    return  quizzes_dict


def __format_to_correct_time(time):
    if ':' not in time:
        if len(time.strip()) == 1:
            return '0' + time.strip() + ':00'
        return time.strip() + ':00'
    hours = time.split(':')[0].strip()
    mins = time.split(':')[1].strip()
    if len(hours) == 1:
        hours = '0' + hours
    if len(mins) == 1:
        mins =  mins + '0'
    return ':'.join([hours,mins])


def __format_time(time_string):
    if len(time_string) <= 2:
        return ''
    start =  __format_to_correct_time(time_string.split('-')[0])
    end =  __format_to_correct_time(time_string.split('-')[1])
    return '-'.join([start,end])


def __get_duration(times_string):
    start =  times_string.split('-')[0]
    end =  times_string.split('-')[1]
    if len(start.split(':')) != 2 or len(end.split(':')) != 2:
        return 0.0
    total_seconds = (datetime(2012, 1, 1, int(end.split(':')[0]), int(end.split(':')[1])) - datetime(2012, 1, 1, int(start.split(':')[0]), int(start.split(':')[1]))).total_seconds()
    return float(total_seconds) /60 /60


def __get_groups_data(soup):
    """return [{"group_num":group_num, "group_data": [{"sub_group_num": sub_group_num, 
                                                        "sub_group_type": sub_group_type,
                                                        "sub_group_lects": {"lect_name":lect_name, "lect_url":lect_url },
                                                        "sub_group_day": sub_group_day,
                                                        "sub_group_time": sub_group_time,
                                                        "sub_group_location": sub_group_location,
                                                        "sub_group_room":, sub_group_room}]},...]"""
    groups_data =[]
    for x in soup.find("table",{"class":"rishum-groups"}).find_all("tr")[1:]:
        group_dict = {} 
        y = x.contents
        group_num = y[0].text
        sub_groups = __get_group_data_list(y[3])
        sub_groups_type = __get_group_data_list(y[4])
        sub_groups_days = __get_group_data_list(y[6])
        sub_groups_time =  __get_group_data_list(y[7])
        sub_groups_location =  __get_group_data_list(y[8])
        sub_groups_room  = __get_group_data_list(y[9])
        sub_groups_lects = __get_group_lect_list(y[5])
        group_dict["group_num"] =  group_num
        group_dict["group_data"] = []
        for sub in zip(sub_groups,sub_groups_type,cycle(sub_groups_lects), sub_groups_days,sub_groups_time,sub_groups_location,sub_groups_room):
            sub_group_dict = {}
            sub_group_dict["sub_group_num"] = sub[0]
            sub_group_dict["sub_group_type"] = sub[1]
            sub_group_dict["sub_group_lects"] = {}
            sub_group_dict["sub_group_lects"]["lect_name"] = sub[2][0]
            sub_group_dict["sub_group_lects"]["lect_url"] = sub[2][1]
            sub_group_dict["sub_group_day"] = sub[3]
            sub_group_dict["sub_group_time"] = __format_time(sub[4])
            sub_group_dict["sub_group_total_hours"] = __get_duration(sub[4])
            sub_group_dict["sub_group_duration"] = sub[5]
            sub_group_dict["sub_group_room"] = sub[6]
            group_dict["group_data"].append(sub_group_dict)
        groups_data.append(group_dict)
    return groups_data
    
def __get_url_soup(url, tries = 0):
    try:
        course_site_html = urlopen(url).read()
    except URLError, IOError:
        tries = tries + 1
        print "Can't access " + url + ", starting try #" + tries
        if tries == 10:
            return None
        return __get_url_soup(url, tries)
    return BeautifulSoup(course_site_html)
    
def __get_complete_json(sem):
    facs = []
    for fac in __get_all_facs_data(sem).items():
        print fac[0]
        if len(fac[0][1]) < 2:
            fac_num = '0' + fac[0][1]
        else:
            fac_num = fac[0][1]
        fac_dict = {"fac_name" : fac[0][0],"fac_num" : fac_num, "fac_courses" : {}}
        for course in fac[1]:
            course_dict = {}
            
            soup = __get_url_soup(COURSE_URL % (course['num'], sem))
            if soup is None:
                continue
            
            course_dict["course_name"] = course['name']
            course_dict["course_num"] = course['num']
            course_dict["course_url"] = __get_course_url(soup)
            course_dict["course_points"] = __get_course_points(soup)
            course_types_hours = __get_course_types_hours(soup)
            course_dict["course_types_hours"] = course_types_hours[:-1]
            course_dict["course_hours"] = course_types_hours[-1]
            course_dict["course_syllabus"] = __get_course_syllabus(soup)
            course_dict["course_prerequisites"] = __get_course_prerequisites(soup)
            course_dict["course_notes"] = __get_course_notes(soup)
            course_dict["course_group"] = __get_groups_data(soup)
            course_dict["faculty"] = fac[0][0]

            test_site =  TEST_URL % (course['num'], sem)
            quiz_site =  QUIZ_URL % (course['num'], sem)
            
            soup_test = __get_url_soup(test_site)
            if soup_test is None:
                course_dict["course_exams"] = None
            else:
                course_dict["course_exams"] = __get_course_exams(soup_test)
    
            soup_quiz = __get_url_soup(quiz_site)
            if soup_quiz is None:
                course_dict["course_quizzes"] = None
            else:
                course_dict["course_quizzes"] = __get_course_exams(soup_quiz)
    
                
            fac_dict["fac_courses"][course['num']] = course_dict
            
        facs.append(fac_dict)
    return facs    

def create_courses_file(sem = "201401"):

    facs = __get_complete_json(sem)
           
    file_path = os.path.join(os.path.split(__file__)[0], "Data_Dumps\\" + sem + ".txt")
    f = open(file_path, "w")
     
    f.write(json.dumps(facs))
    f.close()
    
if __name__ == '__main__':
    create_courses_file()



